# -*- coding: utf-8 -*-
import re
import scrapy
from scrapy_redis.spiders import RedisSpider

import whucs.settings as setting
from ..items import WhuMtwdItem


class LectureSpider(RedisSpider):
    name = "whu_mtwd_detail"

    redis_key = setting.REDIS_MTWD_URLS_KEY

    custom_settings = {
        'ITEM_PIPELINES': {
            'whucs.pipelines.MtwdDetailPipeline': 300
        }
    }

    def make_requests_from_url(self, url):
        return scrapy.Request(url)

    def parse(self, response):
        """
        解析Item
        :param response:
        :return:
        """

        url = response.url

        article = response.xpath("//form[@name='_newscontent_fromname']")

        attributes = article.xpath(".//div[@class='news_attrib']/text()")
        pubdate = attributes.re_first(ur'发布时间：(\d.+:\d+).*')

        print pubdate

        title = article.xpath(".//div[@class='news_title']/text()").extract_first()
        title = re.sub(r'\s', '', title)
        subtitle = article.xpath("//div[@class='news_ytitle']/text()").extract_first()
        if subtitle is not None:
            subtitle = re.sub(r'\s', '', subtitle)
            title = title + ' ' + subtitle
        print title

        content = ""
        ps = article.xpath(".//div[@class='news_content']//p")
        if len(ps) > 0:
            for pp in ps:
                for text in pp.xpath(".//text()").extract():
                    content += text
                content += "\n"
        else:
            texts = response.xpath("//body/div[2]//text()").extract()
            for text in texts:
                content += text
        # print content.decode("unicode_escape").encode('gbk')
        yield WhuMtwdItem(url=url, content=content, time=pubdate, title=title)

